Back to Article
tables_from_markdown.ipynb
Download Notebook
In [2]:
from great_tables import GT, md, html, style, loc, px
import polars as pl
import mdpd
from zfish.plot.plot_commons import cmaps

MONOFONT = "source code pro"
TABLEFONT = None


def gt_render_pdf(
    df,
    header="Base Dimensions",
    hide=None,
    font=MONOFONT,
    weith="normal",
    size=px(14),
    columns=(),
):
    gt_df = (
        GT(df)
        .tab_style(
            style=style.text(
                color=cmaps.spatial_element_dark[3],
                font=MONOFONT,
                weight="normal",
                size=size,
            ),
            locations=loc.body(columns=list(columns)),
        )
        .tab_header(
            title=md(header),
        )
    )
    if hide is not None:
        return gt_df.cols_hide(hide)
    return gt_df


def gt_render(
    df,
    header="Base Dimensions",
    hide=None,
    font=MONOFONT,
    weith="normal",
    size=px(14),
    columns=(),
    v_padding=0.8,
    h_padding=0.8,
):
    gt_df = (
        GT(df)
        .tab_style(
            style=style.text(
                color=cmaps.spatial_element_dark[3],
                font=MONOFONT,
                weight="normal",
                size=size,
            ),
            locations=loc.body(columns=list(columns)),
        )
        .tab_header(
            title=md(header),
        )
        .opt_vertical_padding(scale=v_padding)
        .opt_horizontal_padding(scale=h_padding)
    )
    if hide is not None:
        return gt_df.cols_hide(hide)
    return gt_df


def gt_render_from_markdown(
    df,
    header="Base Dimensions",
    hide=None,
    size=px(14),
    columns=(),
    v_padding=0.8,
    h_padding=0.8,
):
    gt_df = (
        GT(df)
        .tab_header(
            title=md(header),
        )
        .opt_vertical_padding(scale=v_padding)
        .opt_horizontal_padding(scale=h_padding)
    )
    if hide is not None:
        return gt_df.cols_hide(hide)
    return gt_df.fmt_markdown(list(columns))
In [3]:
import polars as pl
import mdpd

df_data_structures = pl.DataFrame(
    mdpd.from_md("""
| Data Structure        | Description |
|------------------------|-----------------------------------------------------------|
| **Raster Data**        |  |
| Image                  | Fluorescence intensity measurements sampled on a regular grid. |
| Binary Mask            | Partition of an image into foreground and background, representing a single object. |
| Label Image            | Partitioning of an image into *n* objects with integer IDs (labels). `0` for background. Represents a collection of objects. |
| Distance Transform     | Image with pixel values storing the distance to a binary mask. |
| Multiscale Pyramid     | Collection of images sampled at different scales (pyramid schedule) with shared coverage. |
| **Vector Data**        |  |
| Point / Vertex         | Position in space, potentially associated with an object or features: `(z, y, x)`. |
| Vector                 | *Difference* between two points, or direction and magnitude: `(dz, dy, dx)`. |
| Line Segment / Edge    | 1D path between two points (also 1-simplex): `(z, y, x), (z, y, x)` *or* `(z, y, x), (dz, dy, dx)`. |
| Face                   | Simple 2D surface, *often* a triangle. Defined by points and their connectivity. |
| Mesh                   | Representation of objects or surfaces using points, edges, and faces placed in physical space. |
| Graph (Spatial)        | Mesh with only points and edges to represent objects in space and their relationships. |
| Neighborhood           | Spatial graph representing object neighborhoods, for example using sparse arrays. |
| **Non-Spatial**\*      |  |
| Feature Table          | Quantitative or categorical features (columns) associated with a set of elements (rows). |
| Graph                  | Nodes and edges representing entities and relationships between them. Many possible representations, *usually* sparse. |
| Hierarchy Table        | Graph encoding relationships (rows) between pairs of objects (columns; 2 IDs). Edge list representation of a graph. |
""")
)


def gt_bold_column_labels(df):
    return (
        GT(df_data_structures)
        .fmt_markdown()
        .cols_label(
            **{col: md(f"**{col}**") for col in df.columns},
        )
    )
In [4]:
gt_bold_column_labels(df_data_structures).tab_source_note(
    source_note=md("*Since all our measurements are derived from pixels they can *always* be associated with spatial objects. But we often drop the spatial data components, like when doing regression modelling with extracted features, and think of them as non-spatial.")
)
Data Structure Description
Raster Data
Image Fluorescence intensity measurements sampled on a regular grid.
Binary Mask Partition of an image into foreground and background, representing a single object.
Label Image Partitioning of an image into n objects with integer IDs (labels). 0 for background. Represents a collection of objects.
Distance Transform Image with pixel values storing the distance to a binary mask.
Multiscale Pyramid Collection of images sampled at different scales (pyramid schedule) with shared coverage.
Vector Data
Point / Vertex Position in space, potentially associated with an object or features: (z, y, x).
Vector Difference between two points, or direction and magnitude: (dz, dy, dx).
Line Segment / Edge 1D path between two points (also 1-simplex): (z, y, x), (z, y, x) or (z, y, x), (dz, dy, dx).
Face Simple 2D surface, often a triangle. Defined by points and their connectivity.
Mesh Representation of objects or surfaces using points, edges, and faces placed in physical space.
Graph (Spatial) Mesh with only points and edges to represent objects in space and their relationships.
Neighborhood Spatial graph representing object neighborhoods, for example using sparse arrays.
Non-Spatial*
Feature Table Quantitative or categorical features (columns) associated with a set of elements (rows).
Graph Nodes and edges representing entities and relationships between them. Many possible representations, usually sparse.
Hierarchy Table Graph encoding relationships (rows) between pairs of objects (columns; 2 IDs). Edge list representation of a graph.
*Since all our measurements are derived from pixels they can always be associated with spatial objects. But we often drop the spatial data components, like when doing regression modelling with extracted features, and think of them as non-spatial.
In [5]:
df_dims_base = pl.DataFrame(
    mdpd.from_md("""
| Type | Dimension | Coordinate | Coordinate Description |
| :---|:--|:-------------|:------------|
| nominal | `roi` | `['site0', 'site1', ..., 'site211']` | Named, bounded regions of space. |
| temporal | `t`  | `[0.0, 15.0, 30.0, ..., 1000.0] s` | Acquisition timestamps. |
| spatial | `z`  | `[0.0, 271.0] μm` | Bounded continuous z-coordinate. |
| spatial | `y`  | `[0.0, 650.0] μm` | Bounded continuous y-coordinate. |
| spatial | `x`  | `[0.0, 650.0] μm` | Bounded continuous x-coordinate. |
| ordinal | `m`   | `[0, 1, 2, 3, 4, 5]` | Sampling scales of the pyramid grids. |
| nominal | `c`   | `['DAPI.0', 'PCNA.0', ..., 'Nanog.3']` | Molecular targets with acquisition cycle. |
| ratio   | `val` | `[0.0, 4.2, 5.4, ...] AU` | Intensity histogram for a channel. |
| nominal | `o`   | `['embryo', 'cell', ..., 'nuc']` | Types of segmented objects. |
| nominal | `lbl`* | `[1, 2, ... n]` | Instance labels of an object type. |
""")
)
In [6]:
tbl = (
    gt_render_from_markdown(df_dims_base, columns=("Dimension", "Coordinate"))
    .cols_move_to_start("Dimension")
    .cols_label({"Dimension": "Dim"})
)
tbl
Base Dimensions
Dim Type Coordinate Coordinate Description
roi nominal ['site0', 'site1', ..., 'site211'] Named, bounded regions of space.
t temporal [0.0, 15.0, 30.0, ..., 1000.0] s Acquisition timestamps.
z spatial [0.0, 271.0] μm Bounded continuous z-coordinate.
y spatial [0.0, 650.0] μm Bounded continuous y-coordinate.
x spatial [0.0, 650.0] μm Bounded continuous x-coordinate.
m ordinal [0, 1, 2, 3, 4, 5] Sampling scales of the pyramid grids.
c nominal ['DAPI.0', 'PCNA.0', ..., 'Nanog.3'] Molecular targets with acquisition cycle.
val ratio [0.0, 4.2, 5.4, ...] AU Intensity histogram for a channel.
o nominal ['embryo', 'cell', ..., 'nuc'] Types of segmented objects.
lbl* nominal [1, 2, ... n] Instance labels of an object type.
In [7]:
df_dims_compound = pl.DataFrame(
    mdpd.from_md("""
| Type        | Dimension      | Components         | Description                                         |
|:------------|:---------------|:-------------------|:----------------------------------------------------|
| categorical | `image`        | `roi`, `m`, `c`    | Single-channel, single-scale image.                 |
| categorical | `label_image`  | `roi`, `m`, `o`    | Single-scale label image.                           |
| categorical | `label_object` | `roi`, `o`, `lbl`  | Segmented object (ID is valid across `m`).          |

""")
)
In [8]:
gt_render_from_markdown(
    df_dims_compound.drop('Type'),
    # hide="Type",
    header="Compound Dimensions",
    columns=("Dimension", "Components"),
    h_padding=1.8,
).cols_move_to_start("Dimension")
Compound Dimensions
Dimension Components Description
image roi, m, c Single-channel, single-scale image.
label_image roi, m, o Single-scale label image.
label_object roi, o, lbl Segmented object (ID is valid across `m`).
In [9]:
df_dims_slice = pl.DataFrame(
    mdpd.from_md("""
| Dimension           | Components                                 | Description                                      |
|---------------------|--------------------------------------------|--------------------------------------------------|
| `mc_image`          | `roi`, `m`, `c:n`                          | Multi-channel, single-scale image.               |
| `image_pyramid`     | `roi`, `m:n`, `c`                          | Single-channel, multi-scale image.               |
| `mc_image_pyramid`  | `roi`, `m:n`, `c:n`                        | Multi-channel, multi-scale image.                |
| `sub_label_image`   | `roi`, `m`, `o`, `lbl:n`                   | Label image formed from a subset of labels.      |
| `voxel_region`      | `roi`, `m`, `iz`, `iy`, `ix`               | Voxel without associated data.                   |
| `mc_mo_voxel`       | `roi`, `m`, `iz`, `iy`, `ix`, `c:n`, `o:n` | Voxel with channel data and object type data.    |
| `roi`               | `roi`, `m:n`, `c:n`, `o:n`                 | Region of interest images and label images.      |
| `dataset`           | `roi:n`, `m:n`, `c:n` `o:n`                | Dataset containing a collection of ROIs.         |
""")
)

df_dims_slice
shape: (8, 3)
Dimension Components Description
str str str
"`mc_image`" "`roi`, `m`, `c:n`" "Multi-channel, single-scale im…
"`image_pyramid`" "`roi`, `m:n`, `c`" "Single-channel, multi-scale im…
"`mc_image_pyramid`" "`roi`, `m:n`, `c:n`" "Multi-channel, multi-scale ima…
"`sub_label_image`" "`roi`, `m`, `o`, `lbl:n`" "Label image formed from a subs…
"`voxel_region`" "`roi`, `m`, `iz`, `iy`, `ix`" "Voxel without associated data."
"`mc_mo_voxel`" "`roi`, `m`, `iz`, `iy`, `ix`, … "Voxel with channel data and ob…
"`roi`" "`roi`, `m:n`, `c:n`, `o:n`" "Region of interest images and …
"`dataset`" "`roi:n`, `m:n`, `c:n` `o:n`" "Dataset containing a collectio…
In [10]:
gt_render_from_markdown(
    df_dims_slice,
    header="Slice Dimensions",
    columns=["Dimension", "Components"],
    h_padding=0.8,
).cols_move_to_start("Dimension")
Slice Dimensions
Dimension Components Description
mc_image roi, m, c:n Multi-channel, single-scale image.
image_pyramid roi, m:n, c Single-channel, multi-scale image.
mc_image_pyramid roi, m:n, c:n Multi-channel, multi-scale image.
sub_label_image roi, m, o, lbl:n Label image formed from a subset of labels.
voxel_region roi, m, iz, iy, ix Voxel without associated data.
mc_mo_voxel roi, m, iz, iy, ix, c:n, o:n Voxel with channel data and object type data.
roi roi, m:n, c:n, o:n Region of interest images and label images.
dataset roi:n, m:n, c:n o:n Dataset containing a collection of ROIs.
In [11]:
df_features_overview = pl.DataFrame(
    mdpd.from_md("""
| Feature Type       | Object ID                     | Resources                                  | Features         | Examples                                         |
|--------------------|-------------------------------|--------------------------------------------|------------------|--------------------------------------------------|
| Label              | `roi`, `o`, `lbl`             | `m`                                        | `f:label`        | Volume, Roundness, Principal Axes, Centroid Position |
| Intensity          | `roi`, `o`, `lbl`             | `m`, `c`                                   | `f:intensity`    | DAPI Abundance*, Concentration**, Variance       |
| Colocalization     | `roi`, `o`, `lbl`             | `m`, `c.0`, `c.1`                          | `f:coloc`        | DAPI Correlation                                 |
| Distance           | `roi`, `o`, `lbl`             | `m`, `o.to`, `[lbl.to]`, `dist_transf`     | `f:distance`     | Centroid Distance To Embryo Border               |
| Hierarchy (1:1)    | `roi`, `o`, `lbl`             | `o.from`, `f:in`                           | `f:in`           | Nuclear DAPI Abundance in Cell                   |
| Hierarchy (m:1)    | `roi`, `o`, `lbl`             | `o.from`, `agg_func`, `[f:in]`             | `Count`, `f:in`  | Mean Nuclear Volume in Embryo                    |
| Neighborhood       | `roi`, `o`, `lbl`             | `nhd_query`, `agg_func`, `[f:in]`          | `Count`, `f:in`  | Cell Count in Radius 30 μm Neighborhood          |
| Embedding          | `roi`, `o`, `lbl`             | `model`, `f:in`                            | `f:out`          | UMAP Embedding, Cell Cycle Phase, Cell Type      |
""") 
)
df_features_overview#.pipe(GT).fmt_markdown().tab_source_note(source_note="*Sum / **Mean intensity")
shape: (8, 5)
Feature Type Object ID Resources Features Examples
str str str str str
"Label" "`roi`, `o`, `lbl`" "`m`" "`f:label`" "Volume, Roundness, Principal A…
"Intensity" "`roi`, `o`, `lbl`" "`m`, `c`" "`f:intensity`" "DAPI Abundance*, Concentration…
"Colocalization" "`roi`, `o`, `lbl`" "`m`, `c.0`, `c.1`" "`f:coloc`" "DAPI Correlation"
"Distance" "`roi`, `o`, `lbl`" "`m`, `o.to`, `[lbl.to]`, `dist… "`f:distance`" "Centroid Distance To Embryo Bo…
"Hierarchy (1:1)" "`roi`, `o`, `lbl`" "`o.from`, `f:in`" "`f:in`" "Nuclear DAPI Abundance in Cell"
"Hierarchy (m:1)" "`roi`, `o`, `lbl`" "`o.from`, `agg_func`, `[f:in]`" "`Count`, `f:in`" "Mean Nuclear Volume in Embryo"
"Neighborhood" "`roi`, `o`, `lbl`" "`nhd_query`, `agg_func`, `[f:i… "`Count`, `f:in`" "Cell Count in Radius 30 μm Nei…
"Embedding" "`roi`, `o`, `lbl`" "`model`, `f:in`" "`f:out`" "UMAP Embedding, Cell Cycle Pha…
In [12]:
gt_render_from_markdown(
    df_features_overview,
    header="Feature Tables",
    columns=("Object ID", "Resources", "Features"),
    # h_padding=0.8,
).tab_source_note(source_note="*Sum / **Mean intensity")
Feature Tables
Feature Type Object ID Resources Features Examples
Label roi, o, lbl m f:label Volume, Roundness, Principal Axes, Centroid Position
Intensity roi, o, lbl m, c f:intensity DAPI Abundance*, Concentration**, Variance
Colocalization roi, o, lbl m, c.0, c.1 f:coloc DAPI Correlation
Distance roi, o, lbl m, o.to, [lbl.to], dist_transf f:distance Centroid Distance To Embryo Border
Hierarchy (1:1) roi, o, lbl o.from, f:in f:in Nuclear DAPI Abundance in Cell
Hierarchy (m:1) roi, o, lbl o.from, agg_func, [f:in] Count, f:in Mean Nuclear Volume in Embryo
Neighborhood roi, o, lbl nhd_query, agg_func, [f:in] Count, f:in Cell Count in Radius 30 μm Neighborhood
Embedding roi, o, lbl model, f:in f:out UMAP Embedding, Cell Cycle Phase, Cell Type
*Sum / **Mean intensity
In [13]:
FEATURE_SETS = {
    "ccp": ("CCP",),
    "ccp-feats": (
        "PCNA",
        "DAPI",
        "n_Volume",
        "n.cy_VolumeRatio",
        "n_Roundness",
        "n_Flatness",
    ),
    "zga": ("Pou5", "Nanog", "H3K27Ac", "H2B"),
    "s5p": ("Pol2.S5P",),
}
(
    GT(
        pl.DataFrame(
            [("", ["Pol2.S2P"])]
            + list(FEATURE_SETS.items())
            + [
                ("ccp-spline", ["bs(CCP, 3, fit_intercept=False)"]),
                ("ccp-asympt", ["CCP(a, b, c)"]),
                ("ccp-power", ["CCP(a, b)"]),
            ],
            orient="row",
            schema=["Component", "Features"],
        )
        .with_columns(
            pl.Series(
                "Description",
                [
                    "Pol-II-S2P (elongating)",
                    "Interphase CCP",
                    "Cell cycle phase features*",
                    "ZGA factors",
                    "Pol-II-S5P (initiating)",
                    "B-spline encoded interphase CCP (3 knots w/o intercept)",
                    "Asymptotic regression on CCP",
                    "Power curve regression on CCP",
                ],
            ),
        )
        .with_columns(pl.col("Features").list.join(", "))
        .with_columns(
            pl.Series(
                "category",
                [
                    "regression target",
                    "linear predictors",
                    "linear predictors",
                    "linear predictors",
                    "linear predictors",
                    "non-linear predictors",
                    "non-linear predictors",
                    "non-linear predictors",
                ],
            )
        )
    )
    .tab_source_note(
        source_note="*a subset of the features that went into the CCP model."
    )
    .tab_stub(rowname_col="Component", groupname_col="category")
    .tab_stubhead("Component")
    .tab_style(
            style=style.text(
                color=cmaps.spatial_element_dark[3],
                font=MONOFONT,
                weight="normal",
            ),
            locations=loc.body(columns=["Component"]),
        )
    .tab_options(column_labels_font_weight="bold")
    # .opt_stylize(style=2, color='blue')
)
Component Features Description
regression target
Pol2.S2P Pol-II-S2P (elongating)
linear predictors
ccp CCP Interphase CCP
ccp-feats PCNA, DAPI, n_Volume, n.cy_VolumeRatio, n_Roundness, n_Flatness Cell cycle phase features*
zga Pou5, Nanog, H3K27Ac, H2B ZGA factors
s5p Pol2.S5P Pol-II-S5P (initiating)
non-linear predictors
ccp-spline bs(CCP, 3, fit_intercept=False) B-spline encoded interphase CCP (3 knots w/o intercept)
ccp-asympt CCP(a, b, c) Asymptotic regression on CCP
ccp-power CCP(a, b) Power curve regression on CCP
*a subset of the features that went into the CCP model.
In [14]:
import polars as pl
from great_tables import GT

df_ccp_components = pl.DataFrame(
    {
        "Component": [
            "`PolarsSelector`",
            "`StratifiedTransformer`",
            "`StratifiedEmbedderAligned`",
            "`CcpTransformer`",
        ],
        "Description": [
            "Selects columns from a Polars DataFrame using a column selector.",
            "Meta-transformer that maps a transformer across subsets of the data by fitting one for each subset.",
            "Meta-transformer that maps an embedding (e.g., UMAP) on subsets of data and aligns the results using a rigid transformation.",
            "Cell cycle phase transformer that takes a pre-embedding as input, fits a principal circle, and returns `CCP`, `NormalizedCCP`, and `DistanceToCircleCCP`."
        ],
    }
)


gt_render_from_markdown(
    df_ccp_components,
    header="Scikit-learn Compatible CCP Components",
    columns=("Component", "Description"),
    h_padding=0.8,
)
Scikit-learn Compatible CCP Components
Component Description
PolarsSelector Selects columns from a Polars DataFrame using a column selector.
StratifiedTransformer Meta-transformer that maps a transformer across subsets of the data by fitting one for each subset.
StratifiedEmbedderAligned Meta-transformer that maps an embedding (e.g., UMAP) on subsets of data and aligns the results using a rigid transformation.
CcpTransformer Cell cycle phase transformer that takes a pre-embedding as input, fits a principal circle, and returns CCP, NormalizedCCP, and DistanceToCircleCCP.
In [22]:
df_ab_stain_conditions
shape: (12, 9)
acquisition wavelength target_primary host_primary dilution_primary secondary_is_fab name_secondary dilution_secondary stain_type
i64 i64 str str str bool str str str
0 488 "Pol-II-S2P" "Rat" "1:250" false "d@rt 488" "1:200" "Antibody"
0 568 "FLAG" "Mouse" "1:500" false "d@ms 568" "1:200" "Antibody"
0 647 "PCNA" "Rabbit" "1:250" false "d@rb 647" "1:200" "Antibody"
1 488 "H3K27Ac" "Rabbit" "1:500" true "Zenon Rb 488" "1:100" "Antibody"
1 568 "bCatenin" "Mouse" "1:250" false "d@ms 568" "1:400" "Antibody"
2 568 "ALYREF" "Mouse" "1:250" true "Zenon Ms 555" "1:100" "Antibody"
2 647 "Pol-II-S5P" "Rat" "1:250" false "d@rt 647" "1:200" "Antibody"
3 488 "Nanog" "Rabbit" "1:280" true "Zenon Rb 488" "1:28" "Antibody"
3 568 "YAP" "Mouse" "1:50" false "d@ms 568" "1:400" "Antibody"
3 647 "XRN2" "Rabbit" "1:100" true "Zenon Rb 647" "1:20" "Antibody"
In [1]:
df_ab_stain_conditions = pl.read_csv("../Data/AB_stainings.csv")

df_ab_stain_conditions.select(
    pl.col("target_primary").alias("primary"),
    pl.col("host_primary").alias("host"),
    pl.col("dilution_primary").alias("dilution"),
    pl.col('name_secondary').alias('secondary'),
    pl.col('dilution_secondary').alias('secondary dilution'),
    pl.when(pl.col('secondary_is_fab')).then(pl.lit('IF direct')).otherwise(pl.lit('IF indirect')).alias('stain type'),

).pipe(GT)
primary host dilution stain type secondary name secondary dilution
Pol-II-S2P Rat 1:250 IF indirect d@rt 488 1:200
FLAG Mouse 1:500 IF indirect d@ms 568 1:200
PCNA Rabbit 1:250 IF indirect d@rb 647 1:200
H3K27Ac Rabbit 1:500 IF direct Zenon Rb 488 1:100
bCatenin Mouse 1:250 IF indirect d@ms 568 1:400
pH3 Rabbit 1:150 IF direct Zenon Rb 647 1:150
H2B Rabbit 1:100 IF direct Zenon Rb 488 1:250
ALYREF Mouse 1:250 IF direct Zenon Ms 555 1:100
Pol-II-S5P Rat 1:250 IF indirect d@rt 647 1:200
Nanog Rabbit 1:280 IF direct Zenon Rb 488 1:28
YAP Mouse 1:50 IF indirect d@ms 568 1:400
XRN2 Rabbit 1:100 IF direct Zenon Rb 647 1:20